<html lang="en">

<head>
	<!-- <link href="./all.css" rel="stylesheet"> -->
	<link href="assets/css/all.min.css" rel="stylesheet">
  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0-alpha2/dist/css/bootstrap.min.css" rel="stylesheet">

  <style>
    body {
      margin: 0;
      height: 100vh;
      display: flex;
      justify-content: center;
      align-items: center;
    }

    /*@font-face {
      font-family: 'FontAwesome';
      src: url('./fa-solid-900.ttf') format('truetype'),
      url('./fa-solid-900.woff2') format('woff2');
      font-weight: normal;
      font-style: normal;
    }*/

    #canvas-wrapper {
      position: relative;
      width: 500px;
      height: 500px;
    }

    #slider-wrapper {
      display: none;
      position: absolute; /* Add this line */
      z-index: 10; /* Add this line */
      top: 47px; /* Adjust this value */
      right: 50px; /* Adjust this value */
    }

    #canvas {
      position: absolute;
      border: 1px solid black;
      width: 100%;
      height: 100%;
    }

    #image-canvas, #temp-canvas {
      position: absolute;
      width: 100%;
      height: 100%;
    }

    .button-wrapper {
      position: absolute;
      top: 0;
      right: 0;
      padding: 10px;
      display: flex;
      flex-direction: column;
      align-items: flex-end;
    }

    .button {
      background-color: #4CAF50;
      border: none;
      border-radius: 8px;
      color: white;
      padding: 4px 8px;
      text-align: center;
      text-decoration: none;
      display: inline-block;
      font-size: 12px;
      margin: 2px 0;
      cursor: pointer;
      width: 32px;
      height: 32px;
    }

    #file-input-wrapper {
      position: absolute;
      top: 0;
      left: 0;
      width: 500px;
      height: 500px;
      cursor: pointer;
    }

    #file-input-wrapper input[type=file] {
      opacity: 0;
      position: absolute;
      top: 0;
      left: 0;
      width: 100%;
      height: 100%;
      cursor: pointer;
    }

    #spinner-container {
      display: flex;
      flex-direction: column;
      align-items: center;
      justify-content: center;
    }
    #spinner-container span {
      font-family: Arial, sans-serif;
      font-size: 16px;
      color: #333;
      margin-top: 10px;
    }

    /* Sparkles */
    .sparkles-container {
      position: absolute;
      top: 0;
      left: 0;
      width: 100%;
      height: 100%;
      overflow: hidden;
      pointer-events: none;
      opacity: 0; /* Hide the sparkles by default */
    }

    .sparkle {
      position: absolute;
      width: 8px;
      height: 8px;
      background-color: #fff;
      border-radius: 50%;
      animation: sparkle 4s linear infinite;
    }

    @keyframes sparkle {
      0% {
        transform: translateX(-10%) scale(0);
        opacity: 0;
      }
      5% {
        opacity: 1;
      }
      100% {
        transform: translateX(110%) scale(1);
        opacity: 0;
      }
    }

  </style>
</head>

<body>
	
	<div id="canvas-wrapper">
	  <canvas id="canvas"></canvas>
    <canvas id="image-canvas"></canvas>
    <canvas id="temp-canvas" style="pointer-events: none;"></canvas>
    <div class="sparkles-container"></div>
	  <div class="button-wrapper">
	  	<button class="button" id="remove-image"><i class="fas fa-times"></i></button>
      <button class="button" id="draw-mask"><i class="fas fa-pencil"></i></button>
      <button class="button" id="magic-tool"><i class="fas fa-wand-magic-sparkles"></i></button>
      <button class="button" id="draw-rect"><i class="fas fa-shapes"></i></button>
      <button class="button" id="seg-everything"><i class="fas fa-fill-drip"></i></button>
      <button class="button" id="undo-path"><i class="fas fa-undo"></i></button>
      <button class="button" id="eraser"><i class="fa-solid fa-eraser"></i></button>
      <button class="button" id="save-images"><i class="fas fa-download"></i></button>
      <button class="button" id="seg-video"><i class="fa-solid fa-film"></i></button>
	  </div>
	  <div id="slider-wrapper">
		  <input type="range" min="1" max="100" value="50" class="slider" id="brush-size">
	  </div>
	  <div id="file-input-wrapper">
	  	<input type="file" id="file-input">
	  </div>
	</div>

  <div id="spinner-container" class="position-fixed top-50 start-50 translate-middle d-none">
      <div class="spinner-border text-primary" role="status"></div>
      <span style="color: #dc3545; font-size: 2em;">Preprocessing image...</span>
  </div>




	<script type="text/javascript">

    document.addEventListener('contextmenu', function(event) {
        event.preventDefault(); // Prevent the default context menu from appearing (right-click menu in browsers)
    });

    async function canvasToBlob(canvas) {
      return new Promise((resolve, reject) => {
        canvas.toBlob((blob) => {
          if (blob) {
              resolve(blob);
          } else {
              reject(new Error("Canvas to Blob conversion failed"));
          }
        });
      });
    }

    async function blobToByteArray(blob) {
      return new Uint8Array(await new Response(blob).arrayBuffer());
    }

    function randomRGBColor() {
      const red = Math.floor(Math.random() * 256);
      const green = Math.floor(Math.random() * 256);
      const blue = Math.floor(Math.random() * 256);
      return { r: red, g: green, b: blue};
    }
    
    // Magic tool functionality
    const tolerance = 30;
    const replacementColor = { r: 147, g: 112, b: 219 };
    const alpha_255 = 191.25;
    const alpha_1 = 1; // for manual drawer (avoid color overlay artifact when = 0.75)
    
    const random_color = {};
    for (let i = 1; i <= 255; i++) {
      random_color[i] = randomRGBColor();
    }

    // canvas image xy to original image xy
    function getXYLocationInOriImage(x, y) {
      x -= scaledX;
      y -= scaledY;

      x /= scaleFactor;
      y /= scaleFactor;

      x = Math.round(x);
      y = Math.round(y);

      return { x, y };
    }

    // canvas idx to original image idx
    function getIdxLocationInOriImage(idx) {
      let x = idx % 500;
      let y = Math.floor(idx / 500);

      x -= scaledX;
      y -= scaledY;

      x /= scaleFactor;
      y /= scaleFactor;

      x = Math.round(x);
      y = Math.round(y);

      return y * originalUnresizedImageData.width + x;
    }

    // scaled image idx to original image idx
    function imgidx2oriidx(idx) {
      let { x, y } = idx2xy(idx);
      x /= scaleFactor;
      y /= scaleFactor;
      x = Math.round(x);
      y = Math.round(y);
      return y * originalUnresizedImageData.width + x;
    }

    // event location is on canvas, which is different from the original image size
    function getEventLocationInOriImage(event) {
      let x = event.offsetX;
      let y = event.offsetY;

      return getXYLocationInOriImage(x, y);
    }

    // // scaled image xy to scaled image idx
    // function xy2idx(x, y) {
    //   return y * scaledWidth + x;
    // }

    // // scaled image idx to scaled image xy
    // function idx2xy(idx) {
    //   let x = idx % scaledWidth;
    //   let y = Math.floor(idx / scaledWidth);

    //   return { x, y };
    // }


    // only seg_everything, rect, and magic tool need to use this function
    function getImagedataFromImageClass(Image, masktype) {
      // Create a original size element and draw the image onto it
      let tmpcanvas = document.createElement('canvas');
      tmpcanvas.width = originalUnresizedImageData.width;
      tmpcanvas.height = originalUnresizedImageData.height;
      let tmpcontext = tmpcanvas.getContext('2d');

      tmpcontext.drawImage(Image, 0, 0);

      // Get the image data from the canvas
      let imageData = tmpcontext.getImageData(0, 0, tmpcanvas.width, tmpcanvas.height);
      let pixelData = imageData.data;

      const imageMask = maskcontext.getImageData(0, 0, maskcanvas.width, maskcanvas.height);
      const data = imageMask.data;

      if (masktype == 'everything') {
        // Get the pixel indices of the mask
        for (let i = 0; i < pixelData.length; i += 4) {
          if (pixelData[i] > 0) {
            data[i] = random_color[pixelData[i]].r; // red
            data[i + 1] = random_color[pixelData[i]].g; // green
            data[i + 2] = random_color[pixelData[i]].b; // blue
            data[i + 3] = alpha_255; // alpha
          }
        }
        maskcontext.putImageData(imageMask, 0, 0);

        oriDrawnPaths.push({
          masks: pixelData,
          type: "everything",
        });
      } 
      else if (masktype == "rect") {
        let pixels = [];
        // Get the pixel indices of the mask
        for (let i = 0; i < pixelData.length; i += 4) {
          if (pixelData[i] == 255 && pixelData[i + 1] == 255 && pixelData[i + 2] == 255) {
            pixels.push(i);
          }
        }

        for (let i = 0; i < pixels.length; i += 1) {
          data[pixels[i]] = replacementColor.r; // red
          data[pixels[i] + 1] = replacementColor.g; // green
          data[pixels[i] + 2] = replacementColor.b; // blue
          data[pixels[i] + 3] = alpha_255; // alpha
        }
        maskcontext.putImageData(imageMask, 0, 0);

        oriDrawnPaths.push({
          points: pixels,
          type: "rect",
        });
      } 
      else if (masktype == "magic") {
        let pixels = [];
        // Get the pixel indices of the mask
        for (let i = 0; i < pixelData.length; i += 4) {
          if (pixelData[i] == 255 && pixelData[i + 1] == 255 && pixelData[i + 2] == 255) {
            pixels.push(i);
          }
        }

        // we don't add anything to the oriDrawnPaths while interactive mode
        oriInteractivePaths.push(pixels);
      }
      else {
        // do nothing
      }

      // Create a canvas size element and draw the image onto it
      tmpcanvas = document.createElement('canvas');
      tmpcanvas.width = canvas.width;
      tmpcanvas.height = canvas.height;
      tmpcontext = tmpcanvas.getContext('2d');

      tmpcontext.drawImage(Image, scaledX, scaledY, scaledWidth, scaledHeight);

      // Get the image data from the canvas
      imageData = tmpcontext.getImageData(0, 0, tmpcanvas.width, tmpcanvas.height);
      pixelData = imageData.data;

      return imageData.data;
    }

    function magicToolHandler(event) {
      if (!isMagicToolActive) return;
      console.log("magic tool handler")

      // step 1: get starting point
      const { x, y } = getEventLocationInOriImage(event);
      promptPoints.push({
          type: (event.which == 1) ? true : false,
          origPoint: [event.offsetX, event.offsetY],
          scaledPoint: [x, y],
      });

      // step 2: get image data for flood fill
      const imageWidth = imageCanvas.width;
      const imageHeight = imageCanvas.height;
      const imageData = imageContext.getImageData(0, 0, imageWidth, imageHeight);
      const pixelData = imageData.data;

      const formData = new FormData();
      const typeList = [];
      const clickList = [];

      for (const thisPrompt of promptPoints) {
          typeList.push(thisPrompt.type ? 1 : 0);
          clickList.push(thisPrompt.scaledPoint[0]);
          clickList.push(thisPrompt.scaledPoint[1]);
      }
      formData.append("type", typeList);
      formData.append("click_list", clickList);
      
      // Send a POST request to the server API
      fetch("/click", {
          method: "POST",
          body: formData,
      })
      .then((response) => response.json())
      .then((data) => {

          let pixels = [];

          // Get the base64-encoded image strings from the JSON response
          const maskBase64 = data.masks;
          const maskImage = new Image();
          maskImage.src = `data:image/png;base64,${maskBase64}`;
          maskImage.onload = function() {
            const pixelData = getImagedataFromImageClass(maskImage, "magic");

            console.log(pixelData.length)
            // Get the pixel indices of the mask
            for (let i = 0; i < pixelData.length; i += 4) {
              if (pixelData[i] == 255 && pixelData[i + 1] == 255 && pixelData[i + 2] == 255) {
                pixels.push(i);
              }
            }
            console.log(pixels.length)

            // step 4: put magic mask on canvas
            redrawPaths(canvas, drawnPaths);
            putDataOnCanvas(canvas, pixels);

            for (const thisPrompt of promptPoints) {
                drawPromptPointOnClick(thisPrompt, canvas);
            }

            // step 5: Add the magic mask to drawnPaths array
            interactivePaths.push(pixels);
          };
      })
      .catch((error) => {
          console.error("Error:", error);
      }).finally(() => {
          // // Hide the spinner
          // spinnerContainer.classList.add("d-none");
      });
    }

    function drawPromptPointOnClick(thisPrompt, canvas) {
      let x = thisPrompt.origPoint[0];
      let y = thisPrompt.origPoint[1];

      const fillColor = `rgba(255, 255, 255, 0.75)`;
      const strokeColor = thisPrompt.type ? `green` : `red`;

      const context = canvas.getContext("2d");

      context.beginPath();
      context.arc(x, y, 3, 0, Math.PI * 2);
      context.fillStyle = fillColor;
      context.fill();
      context.strokeStyle = strokeColor;
      context.stroke();
    }

    function putDataOnCanvas(thisCanvas, pixels) {
      const thisContext = thisCanvas.getContext("2d");
      const canvasData = thisContext.getImageData(
          0,
          0,
          thisCanvas.width,
          thisCanvas.height
      );
      const data = canvasData.data;

      for (let i = 0; i < pixels.length; i += 1) {
          data[pixels[i]] = replacementColor.r; // red
          data[pixels[i] + 1] = replacementColor.g; // green
          data[pixels[i] + 2] = replacementColor.b; // blue
          data[pixels[i] + 3] = alpha_255; // alpha
      }
      thisContext.putImageData(canvasData, 0, 0);
    }

	const fileInput = document.getElementById('file-input');
    let zipfileBase64 = null;
    let imageName = "original_image";
    let videoName = "original_video";
    // for drawer
    const canvas = document.getElementById('canvas');
    const context = canvas.getContext('2d');
    // for image
    const imageCanvas = document.getElementById('image-canvas');
    const imageContext = imageCanvas.getContext('2d');
    imageCanvas.style.zIndex = -1;
    // for rect real-time visualization
    const tempCanvas = document.getElementById('temp-canvas');
    const tempContext = tempCanvas.getContext('2d');
    tempCanvas.style.zIndex = 1;
    // for image-size mask canvas
    const maskcanvas = document.createElement('canvas');
    const maskcontext = maskcanvas.getContext('2d');

    canvas.width = 500;
    canvas.height = 500;
    imageCanvas.width = 500;
    imageCanvas.height = 500;
    tempCanvas.width = 500;
    tempCanvas.height = 500;
    // for saving mask purpose
    let scaledWidth = -1;
    let scaledHeight = -1;
    let scaledX = -1;
    let scaledY = -1;
    // global variables for image
    let scaleFactor = -1;

    let originalUnresizedImageData;
    let originalUnresizedMaskData;

    // Prompt points in magic tool
    const promptPoints = [];
    const interactivePaths = [];
    const oriInteractivePaths = [];

    // Add a new array to store drawn paths
    const drawnPaths = [];
    const oriDrawnPaths = [];
    // for magic tool
    let isMagicToolActive = false;
    // for draw rect
    let isDrawingRect = false;
    let rectStartX = 0;
    let rectStartY = 0;
    // for eraser
    let isErasing = false;
    // for draw mask, keep track of whether user is currently drawing
    let isDrawing = false;
    let isDown = false;
    let lastX, lastY; // keep track of last position of the pointer

    const slider = document.getElementById("brush-size");
    let brushSize = slider.value;

    const removeImageButton = document.getElementById('remove-image');
    const drawMaskButton = document.getElementById('draw-mask');
    const magicToolButton = document.getElementById('magic-tool');
    const fileInputWrapper = document.getElementById('file-input-wrapper');
    const drawRectButton = document.getElementById('draw-rect');

    function stopDrawing() {
      canvas.style.cursor = "auto";
      // stop drawing
      isDrawing = false;
      sliderWrapper.style.display = 'none';
    }
    function stopMagicDrawing() {
      canvas.style.cursor = "auto";
      // stop magic drawing
      try {
        canvas.removeEventListener("mousedown", magicToolHandler);
        console.log('remove magic tool handler')
      } catch (error) {
        console.log(error) // do nothing
      }
      isMagicToolActive = false;

      if (interactivePaths.length != 0) {
          const mask_idx = interactivePaths.length - 1;
          const lastPath = interactivePaths.pop();
          const oriLastPath = oriInteractivePaths.pop();
          promptPoints.splice(0);
          interactivePaths.splice(0);
          oriInteractivePaths.splice(0);

          // draw on scaled-size canvas
          drawnPaths.push({
              points: lastPath,
              type: "magic",
          });
          redrawPaths(canvas, drawnPaths);
          // draw on original-size canvas
          oriDrawnPaths.push({
              points: oriLastPath,
              type: "magic",
          });
          redrawPaths(maskcanvas, oriDrawnPaths);

          const formData = new FormData();
          formData.append("mask_idx", mask_idx);
          fetch("/finish_click", {
              method: "POST",
              body: formData,
          })
              .then((response) => response.json())
              .then((data) => {
                  console.log("Success:", data);
              })
              .catch((error) => {
                  console.error("Error:", error);
              })
              .finally(() => {
                  // // Hide the spinner
                  // spinnerContainer.classList.add("d-none");
              });
      }
    }
    function stopErasing() {
      canvas.style.cursor = "auto";
      // stop erasing
      isErasing = false;
    }

    function stopRecting() {
      canvas.style.cursor = "auto";
      // stop draw recting
      isDrawingRect = false;
    }

    function hideFileInput() {
      fileInputWrapper.style.display = 'none';
    }

    function showFileInput() {
      fileInputWrapper.style.display = 'block';
    }


    // Add event listener for the draw-rect button
    drawRectButton.addEventListener('click', () => {
      // stop any other drawing
      stopDrawing();
      stopMagicDrawing();
      stopErasing();
      // stop drawing
      if (isDrawingRect) {
        canvas.style.cursor = 'auto';
        isDrawingRect = false;
      }
      // start drawing
      else {
        canvas.style.cursor = `crosshair`; // change cursor to a crosshair
        isDrawingRect = true; 
      }
    });


    // for processing video
    async function getFirstFrameAsBlob(file) {
        return new Promise((resolve, reject) => {
          videoName = file.name.split('.')[0];
          console.log(videoName)

          const video = document.createElement('video');
          video.preload = 'auto'; // Preload video
          video.src = URL.createObjectURL(file);

          // Trigger seeked event after metadata loaded
          video.addEventListener('loadedmetadata', function() {
              this.currentTime = 0;
          }, false);

          // Extract first frame when seek operation completes
          video.addEventListener('seeked', function() {
              const canvas = document.createElement('canvas');
              const context = canvas.getContext('2d');
              canvas.width = video.videoWidth;
              canvas.height = video.videoHeight;

              // Draw the first frame of the video onto the canvas
              context.drawImage(video, 0, 0, video.videoWidth, video.videoHeight);

              // Convert the canvas to a blob
              canvas.toBlob(blob => {
                  // Convert blob to file
                  const file = new File([blob], 'frame.png', { type: 'image/png' });
                  resolve(file);
              });

              // Remove the video element after capturing the frame
              video.remove();
          }, false);

          // When video can be played through, upload to server
          video.addEventListener('canplaythrough', function() {
              const spinnerContainer = document.getElementById("spinner-container");
              const spinnerText = spinnerContainer.querySelector('span');
              // Change the spinner text
              spinnerText.textContent = "Uploading video...";
              // Show the spinner
              spinnerContainer.classList.remove("d-none");

              // Create new FormData instance and append the video file
              const formData = new FormData();
              formData.append("video", file);

              // Upload the video file to the server
              fetch("/video", {
                  method: "POST",
                  body: formData,
              })
              .then((response) => response.json())
              .then((data) => {
                  console.log("Success:", data);
              })
              .catch((error) => {
                  console.error("Error:", error);
              })
              .finally(() => {
                  // Hide the spinner
                  spinnerContainer.classList.add("d-none");
                  console.log('finally')
              });
          }, false);
        });
    }

    // for processing image
    function handleImage(file, is_video) {
        // Extract and store the uploaded image name
        imageName = file.name.split('.')[0];
        console.log(imageName)

        const image = new Image();
        image.onload = async () => {
          const spinnerContainer = document.getElementById("spinner-container");
          if (!is_video) {
            const spinnerText = spinnerContainer.querySelector('span');
            // Change the spinner text
            spinnerText.textContent = "Preprocessing image...";
            // Show the spinner
            spinnerContainer.classList.remove("d-none");
          }
          
          // for keeping original sized image
          const tmp_canvas = document.createElement('canvas');
          tmp_canvas.width = image.width;
          tmp_canvas.height = image.height;
          const ctx = tmp_canvas.getContext('2d');
          // for keeping original sized mask
          maskcanvas.width = image.width;
          maskcanvas.height = image.height;

          ctx.drawImage(image, 0, 0, image.width, image.height);
          originalUnresizedImageData = ctx.getImageData(0, 0, image.width, image.height);

          scaleFactor = Math.min(imageCanvas.width / image.width, imageCanvas.height / image.height);

          scaledWidth = image.width * scaleFactor;
          scaledHeight = image.height * scaleFactor;

          scaledX = (imageCanvas.width - scaledWidth) / 2;
          scaledY = (imageCanvas.height - scaledHeight) / 2;

          // reset width and height can refresh the canvas so that prev image will not be kept
          imageCanvas.width = 500;
          imageCanvas.height = 500;
          imageContext.drawImage(image, scaledX, scaledY, scaledWidth, scaledHeight);

          hideFileInput();

          if (is_video) {
              return;
          }
          const imageBlob = await canvasToBlob(tmp_canvas);
          const imageByteArray = await blobToByteArray(imageBlob);

          const formData = new FormData();
          formData.append("image", new Blob([imageByteArray]), "image.png");
          fetch("/image", {
              method: "POST",
              body: formData,
          })
          .then((response) => response.json())
          .then((data) => {
              console.log("Success:", data);
          })
          .catch((error) => {
              console.error("Error:", error);
          }).finally(() => {
            // Hide the spinner
            spinnerContainer.classList.add("d-none");
            console.log('finally')
          });
        };
        image.src = URL.createObjectURL(file);
    }

		fileInput.addEventListener('change', async (event) => {
		  const file = event.target.files[0];
      const fileType = file.type;
      zipfileBase64 = null;

      const validVideoTypes = ['video/mp4', 'video/quicktime'];
      if (validVideoTypes.includes(fileType)) {
          console.log("a video file")
          const firstFrameBlob = await getFirstFrameAsBlob(file);
          console.log(firstFrameBlob);
          handleImage(firstFrameBlob, 1);
      } else {
          console.log(file);
          handleImage(file, 0);
      }
		});

		removeImageButton.addEventListener('click', () => {
			imageContext.clearRect(0, 0, imageCanvas.width, imageCanvas.height); // clear the image canvas
      context.clearRect(0, 0, canvas.width, canvas.height); // clear the drawer canvas
      stopDrawing();
      stopErasing();
      stopMagicDrawing();
      stopRecting();
      
      while (drawnPaths.length > 0) {
        drawnPaths.pop();
      }
      while (oriDrawnPaths.length > 0) {
        oriDrawnPaths.pop();
      }
			showFileInput();
      // Reset the file input value, force the browser to treat the re-uploaded file as a new file
      fileInput.value = '';
		});

    slider.addEventListener("input", () => {
      brushSize = slider.value;
    });

    const sliderWrapper = document.getElementById('slider-wrapper');
    sliderWrapper.style.display = 'none';

		drawMaskButton.addEventListener('click', (event) => {
      stopMagicDrawing();
      stopErasing();
      stopRecting();
      // toggle slider visibility
      sliderWrapper.style.display = (sliderWrapper.style.display == 'none') ? 'block' : 'none';
      // stop drawing
      if (isDrawing) {
        canvas.style.cursor = 'auto';
        isDrawing = false;
      }
      // start drawing
      else {
        canvas.style.cursor = `crosshair`; // change cursor to a crosshair
        isDrawing = true; 
      }
    });

		canvas.addEventListener('mousedown', (event) => {
      if (isDrawing || isErasing) {
        lastX = event.offsetX;
        lastY = event.offsetY;
        isDown = true;

        // Start a new path
        drawnPaths.push({
          type: isErasing ? 'eraser' : 'brush',
          points: [],
          lineWidth: brushSize,
        });
        oriDrawnPaths.push({
          type: isErasing ? 'eraser' : 'brush',
          points: [],
          lineWidth: brushSize / scaleFactor,
        });
      } else if (isDrawingRect) {
        rectStartX = event.offsetX;
        rectStartY = event.offsetY;
        isDown = true;
      }
		});

    canvas.addEventListener('mouseup', () => {
      if (isDrawingRect && isDown) {
        // Clear the temporary canvas
        tempContext.clearRect(0, 0, tempCanvas.width, tempCanvas.height);

        // context.beginPath();
        // context.rect(rectStartX, rectStartY, rectEndX - rectStartX, rectEndY - rectStartY);
        // context.strokeStyle = 'rgba(147, 112, 219, 1)';
        // context.lineWidth = 2;
        // context.stroke();

        // get start and end points of the rect
        const {x: startX, y: startY} = getXYLocationInOriImage(rectStartX, rectStartY);
        const {x: endX, y: endY} = getXYLocationInOriImage(event.offsetX, event.offsetY);
        console.log(rectStartX, rectStartY, event.offsetX, event.offsetY);
        console.log(startX, startY, endX, endY);
        const formData = new FormData();
        formData.append("start_x", startX);
        formData.append("start_y", startY);
        formData.append("end_x", endX);
        formData.append("end_y", endY);
        // Send a POST request to the server API
        fetch("/rect", {
            method: "POST",
            body: formData,
        })
        .then((response) => response.json())
        .then((data) => {
            // console.log("Success:", data.masks);
            let pixels = [];

            // Get the base64-encoded image strings from the JSON response
            const maskBase64 = data.masks;
            const maskImage = new Image();
            maskImage.src = `data:image/png;base64,${maskBase64}`;
            maskImage.onload = function() {
              const pixelData = getImagedataFromImageClass(maskImage, "rect");

              // console.log(pixelData.length)
              // let uniquePixelData = [...new Set(pixelData)];
              // console.log(uniquePixelData);

              // Get the pixel indices of the mask
              for (let i = 0; i < pixelData.length; i += 4) {
                if (pixelData[i] == 255 && pixelData[i + 1] == 255 && pixelData[i + 2] == 255) {
                  pixels.push(i);
                }
              }
              console.log(pixels.length)
              // step 4: put magic mask on canvas
              const canvasData = context.getImageData(0, 0, canvas.width, canvas.height);
              const data = canvasData.data;
              console.log(data.length)
              for (let i = 0; i < pixels.length; i += 1) {
                data[pixels[i]] = replacementColor.r; // red
                data[pixels[i] + 1] = replacementColor.g; // green
                data[pixels[i] + 2] = replacementColor.b; // blue
                data[pixels[i] + 3] = alpha_255; // alpha
              }
              context.putImageData(canvasData, 0, 0);

              // step 5: Add the rect mask to drawnPaths array
              drawnPaths.push({
                points: pixels,
                type: "rect",
              });
            };
        })
        .catch((error) => {
            console.error("Error:", error);
        }).finally(() => {
            // // Hide the spinner
            // spinnerContainer.classList.add("d-none");
        });
      }
      isDown = false;
    });

		canvas.addEventListener('mousemove', (event) => {
      if ((isDrawing || isErasing) && isDown) {
        const x = event.offsetX;
        const y = event.offsetY;

        // Check if the x and y coordinates are within the image boundaries
        if (x >= scaledX && x < scaledX+scaledWidth && y >= scaledY && y < scaledY+scaledHeight &&
            lastX >= scaledX && lastX < scaledX+scaledWidth && lastY >= scaledY && lastY < scaledY+scaledHeight) {
          context.beginPath();
          context.moveTo(lastX, lastY);
          context.lineTo(x, y);
          if (isErasing) {
            context.globalCompositeOperation = 'destination-out';
          } else {
            context.globalCompositeOperation = 'source-over';
          }
          context.strokeStyle = `rgba(147, 112, 219, ${alpha_1})`;
          context.lineWidth = brushSize;
          context.stroke();
          context.beginPath();
          context.arc(x, y, brushSize / 2, 0, 2 * Math.PI);
          context.fillStyle = `rgba(147, 112, 219, ${alpha_1})`;
          context.fill();

          // Add the point to the current path
          const currentPath = drawnPaths[drawnPaths.length - 1];
          currentPath.points.push({ fromX: lastX, fromY: lastY, toX: x, toY: y });
          const oriCurrentPath = oriDrawnPaths[oriDrawnPaths.length - 1];
          const {x: orilastX, y: orilastY} = getXYLocationInOriImage(lastX, lastY);
          const {x: orix, y: oriy} = getXYLocationInOriImage(x, y);
          oriCurrentPath.points.push({ fromX: orilastX, fromY: orilastY, toX: orix, toY: oriy });

          maskcontext.beginPath();
          maskcontext.moveTo(orilastX, orilastY);
          maskcontext.lineTo(orix, oriy);
          if (isErasing) {
            maskcontext.globalCompositeOperation = 'destination-out';
          } else {
            maskcontext.globalCompositeOperation = 'source-over';
          }
          maskcontext.strokeStyle = `rgba(147, 112, 219, ${alpha_1})`;
          maskcontext.lineWidth = brushSize / scaleFactor;
          maskcontext.stroke();
          maskcontext.beginPath();
          maskcontext.arc(orix, oriy, brushSize / scaleFactor / 2, 0, 2 * Math.PI);
          maskcontext.fillStyle = `rgba(147, 112, 219, ${alpha_1})`;
          maskcontext.fill();
        }
        
        lastX = x;
        lastY = y;
      } else if (isDrawingRect && isDown) {
        const x = event.offsetX;
        const y = event.offsetY;

        // Clear the temporary canvas
        tempContext.clearRect(0, 0, tempCanvas.width, tempCanvas.height);

        // Draw the rectangle on the temporary canvas
        tempContext.beginPath();
        tempContext.rect(rectStartX, rectStartY, x - rectStartX, y - rectStartY);
        tempContext.strokeStyle = `rgba(147, 112, 219, ${alpha_1})`;
        tempContext.lineWidth = 2;
        tempContext.stroke();
      }
    });

    const undoPathButton = document.getElementById('undo-path');

    undoPathButton.addEventListener('click', () => {
      if (promptPoints.length > 0) {
          const lastPoint = promptPoints.pop();
          const lastPath = interactivePaths.pop();
          oriInteractivePaths.pop();

          redrawPaths(canvas, drawnPaths);

          const curLen = interactivePaths.length;
          if (curLen > 0) {
              putDataOnCanvas(canvas, interactivePaths[curLen - 1]);
          }

          for (const thisPrompt of promptPoints) {
              drawPromptPointOnClick(thisPrompt, canvas);
          }

      }
      else if (drawnPaths.length > 0) {
        // Remove the last path from the array
        const lastPath = drawnPaths.pop();
        oriDrawnPaths.pop();
        console.log(lastPath)

        if (lastPath.type === 'magic') {
          fetch("/undo", {
              method: "POST",
          })
          .then((response) => response.json())
          .then((data) => {
              console.log("Success:", data);
          })
          .catch((error) => {
              console.error("Error:", error);
          }).finally(() => {
              // // Hide the spinner
              // spinnerContainer.classList.add("d-none");
          });
        } 

        // Clear the canvas
        redrawPaths(canvas, drawnPaths);

        // clear the ori-size mask canvas
        redrawPaths(maskcanvas, oriDrawnPaths);
      }
      else {
      }
    });

    function redrawPaths(thisCanvas, thisDrawnPaths) {
        // Clear the thisCanvas
        const thisContext = thisCanvas.getContext("2d");
        thisContext.clearRect(0, 0, thisCanvas.width, thisCanvas.height);

        // Redraw the remaining paths
        for (const path of thisDrawnPaths) {
          if (path.type === "magic" || path.type === "rect") {
            putDataOnCanvas(thisCanvas, path.points);
          }
          else if (path.type === "everything") {
            console.log("everything")
            const canvasData = thisContext.getImageData(0, 0, thisCanvas.width, thisCanvas.height);
            const data = canvasData.data;

            for (let i = 0; i < path.masks.length; i += 4) {
              if (path.masks[i] > 0) {
                data[i] = random_color[path.masks[i]].r; // red
                data[i + 1] = random_color[path.masks[i]].g; // green
                data[i + 2] = random_color[path.masks[i]].b; // blue
                data[i + 3] = alpha_255; // alpha
              }
            }
            thisContext.putImageData(canvasData, 0, 0);
          }
          else {
            thisContext.lineWidth = path.lineWidth;
            for (const point of path.points) {
              thisContext.beginPath();
              thisContext.moveTo(point.fromX, point.fromY);
              thisContext.lineTo(point.toX, point.toY);
              if (path.type === "eraser") {
                thisContext.globalCompositeOperation = 'destination-out';
              } else {
                thisContext.globalCompositeOperation = 'source-over';
              }
              thisContext.strokeStyle = `rgba(147, 112, 219, ${alpha_1})`;
              thisContext.stroke();
              thisContext.beginPath();
              thisContext.arc(point.toX, point.toY, path.lineWidth / 2, 0, 2 * Math.PI);
              thisContext.fillStyle = `rgba(147, 112, 219, ${alpha_1})`;
              thisContext.fill();
            }
          }
        }
    }

    // magic tool ----------------------------------------------------------------
    // function getPixelIndex(x, y, width) {
    //   return (y * width + x) * 4;
    // }

    // function getColorDifference(color1, color2) {
    //   return Math.abs(color1.r - color2.r) + Math.abs(color1.g - color2.g) + Math.abs(color1.b - color2.b);
    // }

    magicToolButton.addEventListener("click", (event) => {
      stopDrawing();
      stopErasing();
      stopRecting();

      if (!isMagicToolActive) {
        canvas.style.cursor = "crosshair";
        canvas.addEventListener("mousedown", magicToolHandler);
        isMagicToolActive = true;
      } else {
        stopMagicDrawing();
      }
    });

    // save images ----------------------------------------------------------------
    const saveImagesButton = document.getElementById('save-images');

    saveImagesButton.addEventListener('click', () => {
      // stop everything
      stopDrawing();
      stopMagicDrawing();
      stopErasing();
      stopRecting();
      // Save the original sized image
      const tmp_canvas = document.createElement('canvas');
      tmp_canvas.width = originalUnresizedImageData.width;
      tmp_canvas.height = originalUnresizedImageData.height;

      const ctx = tmp_canvas.getContext('2d');
      ctx.putImageData(originalUnresizedImageData, 0, 0);
      const DataUrl = tmp_canvas.toDataURL('image/png');
      const ImageLink = document.createElement('a');
      ImageLink.href = DataUrl;
      ImageLink.download = imageName + '.png';
      ImageLink.click();

      // Save the mask image
      const mask_canvas = document.createElement('canvas');
      mask_canvas.width = originalUnresizedImageData.width;
      mask_canvas.height = originalUnresizedImageData.height;

      const mask_ctx = mask_canvas.getContext('2d');
      // this can resize the image automatically
      mask_ctx.drawImage(canvas, scaledX, scaledY, scaledWidth, scaledHeight, 
                                  0, 0, originalUnresizedImageData.width, originalUnresizedImageData.height);
      // change to binary mask
      const maskdata = mask_ctx.getImageData(0, 0, mask_canvas.width, mask_canvas.height);

      const maskDataUrl = mask_canvas.toDataURL('image/png');
      const maskLink = document.createElement('a');
      maskLink.href = maskDataUrl;
      maskLink.download = imageName + '_mask.png';
      maskLink.click();

      // save the zip file
      if (zipfileBase64 != null) {
        // Decode the base64-encoded data to binary data
        const zipData = atob(zipfileBase64);

        // Convert the decoded data to a Uint8Array
        const byteArray = new Uint8Array(zipData.length);
        for (let i = 0; i < zipData.length; i++) {
          byteArray[i] = zipData.charCodeAt(i);
        }

        // Create a Blob from the byteArray
        const blob = new Blob([byteArray], { type: "application/zip" });

        // Create an anchor element with a download attribute and trigger a click event to download the zip file
        const zipLink = document.createElement('a');
        zipLink.href = URL.createObjectURL(blob);
        zipLink.download = imageName + '.zip';
        zipLink.click();

        // Remove the anchor element after the download is triggered
        setTimeout(() => {
          URL.revokeObjectURL(zipLink.href);
          zipLink.remove();
        }, 0);
      }
    });


    // eraser tool ----------------------------------------------------------------
    const eraserButton = document.getElementById('eraser');

    eraserButton.addEventListener('click', (event) => {
      stopDrawing();
      stopMagicDrawing();
      stopRecting();

      // stop erase drawing
      if (isErasing) {
        canvas.style.cursor = 'auto';
        isErasing = false;
      }
      // start erase
      else {
        canvas.style.cursor = `crosshair`; // change cursor to a crosshair
        isErasing = true; 
      }
    });

    // sparkle
    function createSparkles() {
      const container = document.querySelector(".sparkles-container");
      const numberOfRows = 10;
      const numberOfColumns = 20;
      const rowSpacing = 100 / numberOfRows;
      const columnSpacing = 100 / numberOfColumns;

      for (let i = 0; i < numberOfRows; i++) {
        for (let j = 0; j < numberOfColumns; j++) {
          const sparkle = document.createElement("div");
          sparkle.classList.add("sparkle");
          sparkle.style.top = `${rowSpacing * i + rowSpacing / 2}%`;
          sparkle.style.left = `${columnSpacing * j}%`;
          sparkle.style.animationDuration = `${4}s`;
          sparkle.style.animationDelay = `${(i + j) * 0.1}s`;
          container.appendChild(sparkle);
        }
      }
    }

    function toggleSparkles(visible) {
      const container = document.querySelector(".sparkles-container");
      container.style.opacity = visible ? "1" : "0";
    }

    createSparkles();

    // seg everything ----------------------------------------------------------------
    const segEverythingButton = document.getElementById('seg-everything');

    segEverythingButton.addEventListener('click', () => {
      // stop everything
      stopDrawing();
      stopMagicDrawing();
      stopErasing();
      stopRecting();
      // Show sparkles when processing starts
      toggleSparkles(true);

      // Send the image data to the server
      fetch('/everything', {
        method: 'POST'
      })
      .then((response) => response.json())
      .then((data) => {
        // Get the base64-encoded image strings from the JSON response
        const maskBase64 = data.masks;
        zipfileBase64 = data.zipfile;

        const maskImage = new Image();
        maskImage.src = `data:image/png;base64,${maskBase64}`;
        maskImage.onload = function() {
          const pixelData = getImagedataFromImageClass(maskImage, "everything");

          const canvasData = context.getImageData(0, 0, canvas.width, canvas.height);
          const data = canvasData.data;

          // Get the pixel indices of the mask
          for (let i = 0; i < pixelData.length; i += 4) {
            if (pixelData[i] > 0) {
              data[i] = random_color[pixelData[i]].r; // red
              data[i + 1] = random_color[pixelData[i]].g; // green
              data[i + 2] = random_color[pixelData[i]].b; // blue
              data[i + 3] = alpha_255; // alpha
            }
          }
          
          context.putImageData(canvasData, 0, 0);

          drawnPaths.push({
            masks: pixelData,
            type: "everything",
          });
        };
      }).finally(() => {
        toggleSparkles(false);
      });
    });

    // seg video ----------------------------------------------------------------
    const segVideoButton = document.getElementById('seg-video');

    segVideoButton.addEventListener('click', async () => {
      // stop everything
      stopDrawing();
      stopMagicDrawing();
      stopErasing();
      stopRecting();
      // Show sparkles when processing starts
      const spinnerContainer = document.getElementById("spinner-container");
      const spinnerText = spinnerContainer.querySelector('span');
      // Change the spinner text
      spinnerText.innerHTML = "Preprocessing video...<br>This may take a while<br>depending on video length.";
      // Show the spinner
      spinnerContainer.classList.remove("d-none");

      // const mask_canvas = document.createElement('canvas');
      // mask_canvas.width = originalUnresizedImageData.width;
      // mask_canvas.height = originalUnresizedImageData.height;

      // const mask_ctx = mask_canvas.getContext('2d');
      // // this can resize the image automatically
      // mask_ctx.drawImage(canvas, scaledX, scaledY, scaledWidth, scaledHeight, 
      //                             0, 0, originalUnresizedImageData.width, originalUnresizedImageData.height);

      const maskBlob = await canvasToBlob(maskcanvas);
      const maskByteArray = await blobToByteArray(maskBlob);

      const formData = new FormData();
      formData.append("ini_seg", new Blob([maskByteArray]), "image.png");
      // Send the mask data to the server
      fetch('/ini_seg', {
        method: 'POST',
        body: formData
      })
      .then((response) => {
        // If the response status is 204, throw an error
        if (response.status === 204) {
            window.alert('Video is saved already.');
            throw new Error('Video is saved already');
        }
        return response.blob();
      })
      .then((blob) => {
        // Create a new URL for our video
        var url = URL.createObjectURL(blob);
        var a = document.createElement('a');
        
        // Use the blob URL as the href
        a.href = url;
        
        // Use the Content-Disposition filename if given, otherwise fall back to a default
        a.download = videoName + '.mp4';
        
        // Append the anchor to the body and simulate a click to download the file
        document.body.appendChild(a);
        a.click();
        document.body.removeChild(a);
      })
      .catch((error) => {
        console.error("Error:", error);
      })
      .finally(() => {
        // Hide the spinner
        spinnerContainer.classList.add("d-none");       
      });
    });

	</script>

</body>

</html>
